Skip to content

armgit / hypit

Public Imported from GitHub
0

Clone any viral video with AI agents. Not just a script, the whole workflow: swap the face, the words, the B-roll, ship 100 variants in one command, and get your 100M views.

fix: build a frame tile only for the observer that reads it

rpone9ed1e7c12480
diff --git a/packages/reference-video-tools/src/media.ts b/packages/reference-video-tools/src/media.ts
index 07251958..cecca4c8 100644
--- a/packages/reference-video-tools/src/media.ts
+++ b/packages/reference-video-tools/src/media.ts
@@ -127,7 +127,7 @@ async function shotTile(clip: string, duration: number, target: string): Promise
return target;
}
-export async function prepareMedia(videoPath: string, root: string, duration: number): Promise<{
+export async function prepareMedia(videoPath: string, root: string, duration: number, tiles: boolean): Promise<{
readonly bounds: readonly Bounds[];
readonly storyboard: string;
readonly analysisVideo: string;
@@ -146,7 +146,9 @@ export async function prepareMedia(videoPath: string, root: string, duration: nu
await extract(videoPath, ["-i", videoPath, "-ss", String(bound.start), "-t", String(Math.max(0.1, bound.end - bound.start)), "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-c:a", "aac", "-b:a", "96k", "-movflags", "+faststart"], clip);
await extract(videoPath, ["-i", videoPath, "-ss", String(bound.start + (bound.end - bound.start) / 2), "-frames:v", "1", "-vf", "scale='min(720,iw)':-2", "-q:v", "3"], join(shotDir, `${id}-representative.jpg`));
await extract(videoPath, ["-sseof", "-0.1", "-i", clip, "-update", "1", "-frames:v", "1", "-vf", "scale='min(720,iw)':-2", "-q:v", "3"], join(shotDir, `${id}-tail.jpg`));
- await shotTile(clip, bound.end - bound.start, join(shotDir, `${id}-frames.jpg`));
+ // Only the observer that reads pictures has anything to read them from, and building a tile per
+ // shot is a decode per shot. The other observer is handed the clip itself.
+ if (tiles) await shotTile(clip, bound.end - bound.start, join(shotDir, `${id}-frames.jpg`));
if (await hasAudio(clip)) await extract(videoPath, ["-sseof", "-3", "-i", clip, "-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000", "-c:a", "pcm_s16le"], join(shotDir, `${id}-audio.wav`));
}));
const inputs = bounds.map((_, index) => `[${index}:v]`).join("");
diff --git a/packages/reference-video-tools/src/tools.ts b/packages/reference-video-tools/src/tools.ts
index c24b17ef..4fd7a2a4 100644
--- a/packages/reference-video-tools/src/tools.ts
+++ b/packages/reference-video-tools/src/tools.ts
@@ -178,12 +178,17 @@ function asPictures(media: readonly string[], state: ReferenceState): readonly s
const pictures: string[] = [];
for (const path of media) {
const tile = tiles.get(path);
- if (tile !== undefined) pictures.push(tile);
+ if (tile !== undefined) {
+ // Every shot of a reference prepared for this observer has a tile. One without means the media
+ // was prepared for the observer that reads video, which `prepare_reference` refuses to mix.
+ assert(tile !== null, `shot media for ${path} has no frame tile; re-prepare the reference with --redo all --observer agent`);
+ pictures.push(tile);
+ }
// A whole-reference question is asked over the whole reference. The storyboard puts every shot in
// one picture and is how the video is read at a glance, but one frame per shot answers neither
// what moves nor what recurs, so the shot tiles come with it and the question sees every frame
// the shot observations see.
- else if (path === state.analysis_video_ref) pictures.push(state.storyboard_ref, ...state.shots.map((shot) => shot.frames_tile_ref));
+ else if (path === state.analysis_video_ref) pictures.push(state.storyboard_ref, ...state.shots.flatMap((shot) => shot.frames_tile_ref === null ? [] : [shot.frames_tile_ref]));
else if (!path.toLowerCase().endsWith(".wav")) pictures.push(path);
}
return [...new Set(pictures)];
@@ -259,7 +264,7 @@ async function runObservationTasks(
return new Map(tasks.map((task) => [task.key, cache[task.key] ?? answers.get(task.key) ?? observation("failed", "not observed")]));
}
-async function shotFromBound(root: string, index: number, bound: { start: number; end: number; group: number; part: number; parts: number }): Promise<Shot> {
+async function shotFromBound(root: string, index: number, bound: { start: number; end: number; group: number; part: number; parts: number }, tiles: boolean): Promise<Shot> {
const id = String(index + 1).padStart(3, "0");
const dir = join(root, "shots");
const audioPath = join(dir, `${id}-audio.wav`);
@@ -277,7 +282,7 @@ async function shotFromBound(root: string, index: number, bound: { start: number
clip_ref: join(dir, `${id}.mp4`),
representative_frame_ref: join(dir, `${id}-representative.jpg`),
tail_frame_ref: join(dir, `${id}-tail.jpg`),
- frames_tile_ref: join(dir, `${id}-frames.jpg`),
+ frames_tile_ref: tiles ? join(dir, `${id}-frames.jpg`) : null,
audio_tail_ref,
};
}
@@ -438,8 +443,8 @@ export function createReferenceVideoTools(options: ToolOptions = {}): ReferenceV
state = existing;
analysisVideo = existing.analysis_video_ref;
} else {
- const media = await prepareMedia(videoPath, root, info.duration);
- const shots = await Promise.all(media.bounds.map((bound, index) => shotFromBound(root, index, bound)));
+ const media = await prepareMedia(videoPath, root, info.duration, observer === "agent");
+ const shots = await Promise.all(media.bounds.map((bound, index) => shotFromBound(root, index, bound, observer === "agent")));
state = {
reference_id: reference,
video_path: videoPath,
diff --git a/packages/reference-video-tools/src/types.ts b/packages/reference-video-tools/src/types.ts
index 7acb16ae..c17d8f0f 100644
--- a/packages/reference-video-tools/src/types.ts
+++ b/packages/reference-video-tools/src/types.ts
@@ -45,8 +45,11 @@ export type Shot = {
readonly clip_ref: string;
readonly representative_frame_ref: string;
readonly tail_frame_ref: string;
- /** The shot's frames sampled evenly and tiled into one picture, in reading order. */
- readonly frames_tile_ref: string;
+ /**
+ * The shot's frames sampled evenly and tiled into one picture, in reading order. Built for the
+ * `agent` observer, which reads it in place of the clip, and null for the observer that reads video.
+ */
+ readonly frames_tile_ref: string | null;
readonly audio_tail_ref: string | null;
};