Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.
TypeScript agentic-aiaiai-agentsai-videocompilerdslffmpeggenerative-aillmmarkup-languagemonorepoplugin-systemprogramming-languagetext-to-videotypescriptvideovideo-automationvideo-clonevideo-editingvideo-generation
fix: build a frame tile only for the observer that reads it
rpone9ed1e7c12480
diff --git a/packages/reference-video-tools/src/media.ts b/packages/reference-video-tools/src/media.tsindex 07251958..cecca4c8 100644--- a/packages/reference-video-tools/src/media.ts+++ b/packages/reference-video-tools/src/media.ts@@ -127,7 +127,7 @@ async function shotTile(clip: string, duration: number, target: string): Promisereturn target;}-export async function prepareMedia(videoPath: string, root: string, duration: number): Promise<{+export async function prepareMedia(videoPath: string, root: string, duration: number, tiles: boolean): Promise<{readonly bounds: readonly Bounds[];readonly storyboard: string;readonly analysisVideo: string;@@ -146,7 +146,9 @@ export async function prepareMedia(videoPath: string, root: string, duration: nuawait extract(videoPath, ["-i", videoPath, "-ss", String(bound.start), "-t", String(Math.max(0.1, bound.end - bound.start)), "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart"], clip);await extract(videoPath, ["-i", videoPath, "-ss", String(bound.start + (bound.end - bound.start) / 2), "-frames:v", "1", "-vf", "scale='min(720,iw)':-2", "-q:v", "3"], join(shotDir, `${id}-representative.jpg`));await extract(videoPath, ["-sseof", "-0.1", "-i", clip, "-update", "1", "-frames:v", "1", "-vf", "scale='min(720,iw)':-2", "-q:v", "3"], join(shotDir, `${id}-tail.jpg`));- await shotTile(clip, bound.end - bound.start, join(shotDir, `${id}-frames.jpg`));+ // Only the observer that reads pictures has anything to read them from, and building a tile per+ // shot is a decode per shot. The other observer is handed the clip itself.+ if (tiles) await shotTile(clip, bound.end - bound.start, join(shotDir, `${id}-frames.jpg`));if (await hasAudio(clip)) await extract(videoPath, ["-sseof", "-3", "-i", clip, "-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le"], join(shotDir, `${id}-audio.wav`));}));const inputs = bounds.map((_, index) => `[${index}:v]`).join("");diff --git a/packages/reference-video-tools/src/tools.ts b/packages/reference-video-tools/src/tools.tsindex c24b17ef..4fd7a2a4 100644--- a/packages/reference-video-tools/src/tools.ts+++ b/packages/reference-video-tools/src/tools.ts@@ -178,12 +178,17 @@ function asPictures(media: readonly string[], state: ReferenceState): readonly sconst pictures: string[] = [];for (const path of media) {const tile = tiles.get(path);- if (tile !== undefined) pictures.push(tile);+ if (tile !== undefined) {+ // Every shot of a reference prepared for this observer has a tile. One without means the media+ // was prepared for the observer that reads video, which `prepare_reference` refuses to mix.+ assert(tile !== null, `shot media for ${path} has no frame tile; re-prepare the reference with --redo all --observer agent`);+ pictures.push(tile);+ }// A whole-reference question is asked over the whole reference. The storyboard puts every shot in// one picture and is how the video is read at a glance, but one frame per shot answers neither// what moves nor what recurs, so the shot tiles come with it and the question sees every frame// the shot observations see.- else if (path === state.analysis_video_ref) pictures.push(state.storyboard_ref, ...state.shots.map((shot) => shot.frames_tile_ref));+ else if (path === state.analysis_video_ref) pictures.push(state.storyboard_ref, ...state.shots.flatMap((shot) => shot.frames_tile_ref === null ? [] : [shot.frames_tile_ref]));else if (!path.toLowerCase().endsWith(".wav")) pictures.push(path);}return [...new Set(pictures)];@@ -259,7 +264,7 @@ async function runObservationTasks(return new Map(tasks.map((task) => [task.key, cache[task.key] ?? answers.get(task.key) ?? observation("failed", "not observed")]));}-async function shotFromBound(root: string, index: number, bound: { start: number; end: number; group: number; part: number; parts: number }): Promise<Shot> {+async function shotFromBound(root: string, index: number, bound: { start: number; end: number; group: number; part: number; parts: number }, tiles: boolean): Promise<Shot> {const id = String(index + 1).padStart(3, "0");const dir = join(root, "shots");const audioPath = join(dir, `${id}-audio.wav`);@@ -277,7 +282,7 @@ async function shotFromBound(root: string, index: number, bound: { start: numberclip_ref: join(dir, `${id}.mp4`),representative_frame_ref: join(dir, `${id}-representative.jpg`),tail_frame_ref: join(dir, `${id}-tail.jpg`),- frames_tile_ref: join(dir, `${id}-frames.jpg`),+ frames_tile_ref: tiles ? join(dir, `${id}-frames.jpg`) : null,audio_tail_ref,};}@@ -438,8 +443,8 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceVstate = existing;analysisVideo = existing.analysis_video_ref;} else {- const media = await prepareMedia(videoPath, root, info.duration);- const shots = await Promise.all(media.bounds.map((bound, index) => shotFromBound(root, index, bound)));+ const media = await prepareMedia(videoPath, root, info.duration, observer === "agent");+ const shots = await Promise.all(media.bounds.map((bound, index) => shotFromBound(root, index, bound, observer === "agent")));state = {reference_id: reference,video_path: videoPath,diff --git a/packages/reference-video-tools/src/types.ts b/packages/reference-video-tools/src/types.tsindex 7acb16ae..c17d8f0f 100644--- a/packages/reference-video-tools/src/types.ts+++ b/packages/reference-video-tools/src/types.ts@@ -45,8 +45,11 @@ export type Shot = {readonly clip_ref: string;readonly representative_frame_ref: string;readonly tail_frame_ref: string;- /** The shot's frames sampled evenly and tiled into one picture, in reading order. */- readonly frames_tile_ref: string;+ /**+ * The shot's frames sampled evenly and tiled into one picture, in reading order. Built for the+ * `agent` observer, which reads it in place of the clip, and null for the observer that reads video.+ */+ readonly frames_tile_ref: string | null;readonly audio_tail_ref: string | null;};